{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from statsmodels.tsa.stattools import grangercausalitytests as Granger\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['/mnt/c/Users/tiago/Documents/Uni/anm/anm-project/data/training_data/host/db_oracle_11g.csv',\n",
       " '/mnt/c/Users/tiago/Documents/Uni/anm/anm-project/data/training_data/host/dcos_container.csv',\n",
       " '/mnt/c/Users/tiago/Documents/Uni/anm/anm-project/data/training_data/host/dcos_docker.csv',\n",
       " '/mnt/c/Users/tiago/Documents/Uni/anm/anm-project/data/training_data/host/mw_redis.csv',\n",
       " '/mnt/c/Users/tiago/Documents/Uni/anm/anm-project/data/training_data/host/os_linux.csv']"
      ]
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "KPI_DIR = \"/mnt/c/Users/tiago/Documents/Uni/anm/anm-project/data/training_data/host/\"\n",
    "KPI_FILES = [KPI_DIR+k for k in os.listdir(KPI_DIR)]\n",
    "KPI_FILES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                  itemid                  name     bomc_id  \\\n",
       "0        999999998651280          CPU_free_pct  ZJ-002-056   \n",
       "0               63309300   Sent_errors_packets  ZJ-001-070   \n",
       "1        999999996431400  Processor_load_5_min  ZJ-001-005   \n",
       "2        999999996432240           Disk_rd_ios  ZJ-001-098   \n",
       "3        999999996437700          CPU_util_pct  ZJ-001-001   \n",
       "...                  ...                   ...         ...   \n",
       "380143   999999996508399          used_cpu_sys  ZJ-005-022   \n",
       "380144   999999996508394       Redis_key_count  ZJ-005-033   \n",
       "380145   999999996508393            redis_load  ZJ-005-035   \n",
       "1214597  999999996432179       CPU_iowait_time  ZJ-001-010   \n",
       "1214607         65439899          Disk_io_util  ZJ-001-125   \n",
       "\n",
       "                  timestamp         value    cmdb_id  \n",
       "0       2020-05-03 17:00:00     98.119746     db_008  \n",
       "0       2020-05-03 17:00:00      0.000000     os_012  \n",
       "1       2020-05-03 17:00:00      0.011667     os_012  \n",
       "2       2020-05-03 17:00:00      0.000000     os_007  \n",
       "3       2020-05-03 17:00:00      1.750118     os_002  \n",
       "...                     ...           ...        ...  \n",
       "380143  2020-05-04 16:59:59      8.440000  redis_001  \n",
       "380144  2020-05-04 16:59:59  19799.000000  redis_001  \n",
       "380145  2020-05-04 16:59:59      0.000000  redis_001  \n",
       "1214597 2020-05-04 16:59:59      0.002779     os_007  \n",
       "1214607 2020-05-04 16:59:59      0.000000     os_017  \n",
       "\n",
       "[2387373 rows x 6 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>itemid</th>\n      <th>name</th>\n      <th>bomc_id</th>\n      <th>timestamp</th>\n      <th>value</th>\n      <th>cmdb_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>999999998651280</td>\n      <td>CPU_free_pct</td>\n      <td>ZJ-002-056</td>\n      <td>2020-05-03 17:00:00</td>\n      <td>98.119746</td>\n      <td>db_008</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>63309300</td>\n      <td>Sent_errors_packets</td>\n      <td>ZJ-001-070</td>\n      <td>2020-05-03 17:00:00</td>\n      <td>0.000000</td>\n      <td>os_012</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>999999996431400</td>\n      <td>Processor_load_5_min</td>\n      <td>ZJ-001-005</td>\n      <td>2020-05-03 17:00:00</td>\n      <td>0.011667</td>\n      <td>os_012</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>999999996432240</td>\n      <td>Disk_rd_ios</td>\n      <td>ZJ-001-098</td>\n      <td>2020-05-03 17:00:00</td>\n      <td>0.000000</td>\n      <td>os_007</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>999999996437700</td>\n      <td>CPU_util_pct</td>\n      <td>ZJ-001-001</td>\n      <td>2020-05-03 17:00:00</td>\n      <td>1.750118</td>\n      <td>os_002</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>380143</th>\n      <td>999999996508399</td>\n      <td>used_cpu_sys</td>\n      <td>ZJ-005-022</td>\n      <td>2020-05-04 16:59:59</td>\n      <td>8.440000</td>\n      <td>redis_001</td>\n    </tr>\n    <tr>\n      <th>380144</th>\n      <td>999999996508394</td>\n      <td>Redis_key_count</td>\n      <td>ZJ-005-033</td>\n      <td>2020-05-04 16:59:59</td>\n      <td>19799.000000</td>\n      <td>redis_001</td>\n    </tr>\n    <tr>\n      <th>380145</th>\n      <td>999999996508393</td>\n      <td>redis_load</td>\n      <td>ZJ-005-035</td>\n      <td>2020-05-04 16:59:59</td>\n      <td>0.000000</td>\n      <td>redis_001</td>\n    </tr>\n    <tr>\n      <th>1214597</th>\n      <td>999999996432179</td>\n      <td>CPU_iowait_time</td>\n      <td>ZJ-001-010</td>\n      <td>2020-05-04 16:59:59</td>\n      <td>0.002779</td>\n      <td>os_007</td>\n    </tr>\n    <tr>\n      <th>1214607</th>\n      <td>65439899</td>\n      <td>Disk_io_util</td>\n      <td>ZJ-001-125</td>\n      <td>2020-05-04 16:59:59</td>\n      <td>0.000000</td>\n      <td>os_017</td>\n    </tr>\n  </tbody>\n</table>\n<p>2387373 rows × 6 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "kpis_df = pd.concat([pd.read_csv(f) for f in KPI_FILES]).sort_values('timestamp')\n",
    "kpis_df['timestamp'] = kpis_df['timestamp'].apply(lambda x: datetime.datetime.fromtimestamp(x/1000.0))\n",
    "kpis_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "dict_keys(['container_001', 'container_002', 'db_001', 'db_002', 'db_003', 'db_004', 'db_005', 'db_006', 'db_007', 'db_008', 'db_009', 'db_010', 'db_011', 'db_012', 'db_013', 'docker_001', 'docker_002', 'docker_003', 'docker_004', 'docker_005', 'docker_006', 'docker_007', 'docker_008', 'os_001', 'os_002', 'os_003', 'os_004', 'os_005', 'os_006', 'os_007', 'os_008', 'os_009', 'os_010', 'os_011', 'os_012', 'os_013', 'os_014', 'os_015', 'os_016', 'os_017', 'os_018', 'os_019', 'os_020', 'os_021', 'os_022', 'redis_001', 'redis_002', 'redis_003', 'redis_004', 'redis_005', 'redis_006', 'redis_007', 'redis_008', 'redis_009', 'redis_010', 'redis_011', 'redis_012'])"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "kpis = dict(tuple(kpis_df.groupby('cmdb_id')))\n",
    "kpis.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = []\n",
    "for key in kpis:\n",
    "    names = kpis[key]['name'].unique()\n",
    "    nodes.extend([(key, name) for name in names])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "('container_001', 'container_fail_percent')"
      ]
     },
     "metadata": {},
     "execution_count": 204
    }
   ],
   "source": [
    "nodes[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "causality_graph = nx.DiGraph()\n",
    "causality_graph.add_nodes_from(map(lambda x: f'{x[0]}:{x[1]}', nodes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = list(map(lambda x: f'{x[0]}:{x[1]}', nodes))\n",
    "edge_pairs = [(i,j) for index, i in enumerate(a) for j in a[index+1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "2143485"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "len(edge_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_values(node):\n",
    "    host, name = node.split(':')\n",
    "    return kpis[host][kpis[host]['name'] == name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "formatted_df = {}\n",
    "for host in list(kpis.keys()):\n",
    "    groups = list(kpis[host].groupby('name'))\n",
    "    for i in groups:\n",
    "        formatted_df[f'{host}:{i[0]}'] = scaler.fit_transform(i[1]['value'].reset_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[0.00000000e+00, 0.00000000e+00],\n",
       "       [6.97107006e-04, 0.00000000e+00],\n",
       "       [1.04566051e-03, 0.00000000e+00],\n",
       "       ...,\n",
       "       [9.98954339e-01, 0.00000000e+00],\n",
       "       [9.99651446e-01, 0.00000000e+00],\n",
       "       [1.00000000e+00, 0.00000000e+00]])"
      ]
     },
     "metadata": {},
     "execution_count": 42
    }
   ],
   "source": [
    "formatted_df['container_001:container_fail_percent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dtw import dtw, accelerated_dtw\n",
    "from dtaidistance import dtw\n",
    "from fastdtw import fastdtw\n",
    "from scipy.spatial.distance import euclidean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "hosts = list(formatted_df.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'container_001:container_fail_percent'"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "hosts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Currently at  0\n",
      "Currently at  100\n",
      "Currently at  200\n",
      "Currently at  300\n",
      "Currently at  400\n",
      "Currently at  500\n",
      "Currently at  600\n",
      "Currently at  700\n",
      "Currently at  800\n",
      "Currently at  900\n",
      "Currently at  1000\n",
      "Currently at  1100\n",
      "Currently at  1200\n",
      "Currently at  1300\n",
      "Currently at  1400\n",
      "Currently at  1500\n",
      "Currently at  1600\n",
      "Currently at  1700\n",
      "Currently at  1800\n",
      "Currently at  1900\n",
      "Currently at  2000\n"
     ]
    }
   ],
   "source": [
    "base_dst = {}\n",
    "\n",
    "start = hosts[0]\n",
    "start_df = formatted_df[start]\n",
    "for i, next_host in enumerate(hosts[1:]):\n",
    "    if i % 100 == 0:\n",
    "        print('Currently at ', i)\n",
    "    next_df = formatted_df[next_host]\n",
    "    base_dst[(start,next_host)] = fastdtw(start_df, next_df, dist=euclidean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "distances = {}\n",
    "for edge in edge_pairs:\n",
    "    src,dst = edge\n",
    "    if src == start:\n",
    "        distances[edge] = base_dst[edge][0]\n",
    "    else:\n",
    "        distances[edge] = abs(base_dst[(start, src)][0] - base_dst[(start, dst)][0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[('container_001:container_fail_percent',\n",
       "  'container_002:container_fail_percent'),\n",
       " ('container_001:container_fail_percent', 'db_001:CPU_free_pct'),\n",
       " ('container_001:container_fail_percent', 'db_001:Hang'),\n",
       " ('container_001:container_fail_percent', 'db_001:DbTime'),\n",
       " ('container_001:container_fail_percent', 'db_001:Row_Lock')]"
      ]
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "edge_pairs[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.array(list(distances.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.array(list(map(lambda x: round(x, 3), arr)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "11886"
      ]
     },
     "metadata": {},
     "execution_count": 83
    }
   ],
   "source": [
    "len(np.where(arr == 0)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('weights.pickle', 'wb') as f:\n",
    "    pickle.dump(distances, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in distances:\n",
    "    val = round(distances[key],3)\n",
    "    if (val == 0):\n",
    "        continue\n",
    "    src, dst = key\n",
    "    causality_graph.add_edge(src, dst, weight = 1.0/distances[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('graph.pickle', 'wb') as f:\n",
    "    pickle.dump(causality_graph, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "1440.0\n255.19999999999453\n1188.999999999983\n1440.0\n25550.600000000057\n24110.600000000053\n"
     ]
    }
   ],
   "source": [
    "v1 = formatted_df['container_001:container_fail_percent']['value']\n",
    "v2 = formatted_df['db_001:Hang']['value']\n",
    "v3 = formatted_df['db_001:DbTime']['value']\n",
    "v4 = formatted_df['db_001:Sess_Active']['value']\n",
    "print(fastdtw(v1, v2)[0])\n",
    "print(fastdtw(v1, v4)[0])\n",
    "print(fastdtw(v2, v4)[0])\n",
    "\n",
    "print(fastdtw(v1, v2)[0])\n",
    "print(fastdtw(v1, v3)[0])\n",
    "print(fastdtw(v2, v3)[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "os_013:Processor_load_15_min', 'os_013:Processor_load_1_min', 'os_013:Processor_load_5_min', 'os_013:Received_errors_packets', 'os_013:Received_packets', 'os_013:Received_queue', 'os_013:Recv_total', 'os_013:Send_total', 'os_013:Sent_errors_packets', 'os_013:Sent_packets', 'os_013:Sent_queue', 'os_013:Shared_memory', 'os_013:Swap_used_pct', 'os_013:System_block_queue_length', 'os_013:System_wait_queue_length', 'os_013:Zombie_Process', 'os_013:ss_total', 'os_014:Agent_ping', 'os_014:Buffers_used', 'os_014:CPU_idle_pct', 'os_014:CPU_iowait_time', 'os_014:CPU_system_time', 'os_014:CPU_user_time', 'os_014:CPU_util_pct', 'os_014:Cache_used', 'os_014:Disk_avgqu_sz', 'os_014:Disk_await', 'os_014:Disk_io_util', 'os_014:Disk_rd_ios', 'os_014:Disk_rd_kbs', 'os_014:Disk_svctm', 'os_014:Disk_wr_ios', 'os_014:Disk_wr_kbs', 'os_014:FS_max_avail', 'os_014:FS_max_util', 'os_014:FS_total_space', 'os_014:FS_used_pct', 'os_014:FS_used_space', 'os_014:ICMP_ping', 'os_014:Incoming_network_traffic', 'os_014:Memory_available', 'os_014:Memory_available_pct', 'os_014:Memory_free', 'os_014:Memory_total', 'os_014:Memory_used', 'os_014:Memory_used_pct', 'os_014:Num_of_processes', 'os_014:Num_of_running_processes', 'os_014:Outgoing_network_traffic', 'os_014:Page_pi', 'os_014:Page_po', 'os_014:Processor_load_15_min', 'os_014:Processor_load_1_min', 'os_014:Processor_load_5_min', 'os_014:Received_errors_packets', 'os_014:Received_packets', 'os_014:Received_queue', 'os_014:Recv_total', 'os_014:Send_total', 'os_014:Sent_errors_packets', 'os_014:Sent_packets', 'os_014:Sent_queue', 'os_014:Shared_memory', 'os_014:Swap_used_pct', 'os_014:System_block_queue_length', 'os_014:System_wait_queue_length', 'os_014:Zombie_Process', 'os_014:ss_total', 'os_015:Agent_ping', 'os_015:Buffers_used', 'os_015:CPU_idle_pct', 'os_015:CPU_iowait_time', 'os_015:CPU_system_time', 'os_015:CPU_user_time', 'os_015:CPU_util_pct', 'os_015:Cache_used', 'os_015:Disk_avgqu_sz', 'os_015:Disk_await', 'os_015:Disk_io_util', 'os_015:Disk_rd_ios', 'os_015:Disk_rd_kbs', 'os_015:Disk_svctm', 'os_015:Disk_wr_ios', 'os_015:Disk_wr_kbs', 'os_015:FS_max_avail', 'os_015:FS_max_util', 'os_015:FS_total_space', 'os_015:FS_used_pct', 'os_015:FS_used_space', 'os_015:ICMP_ping', 'os_015:Incoming_network_traffic', 'os_015:Memory_available', 'os_015:Memory_available_pct', 'os_015:Memory_free', 'os_015:Memory_total', 'os_015:Memory_used', 'os_015:Memory_used_pct', 'os_015:Num_of_processes', 'os_015:Num_of_running_processes', 'os_015:Outgoing_network_traffic', 'os_015:Page_pi', 'os_015:Page_po', 'os_015:Processor_load_15_min', 'os_015:Processor_load_1_min', 'os_015:Processor_load_5_min', 'os_015:Received_errors_packets', 'os_015:Received_packets', 'os_015:Received_queue', 'os_015:Recv_total', 'os_015:Send_total', 'os_015:Sent_errors_packets', 'os_015:Sent_packets', 'os_015:Sent_queue', 'os_015:Shared_memory', 'os_015:Swap_used_pct', 'os_015:System_block_queue_length', 'os_015:System_wait_queue_length', 'os_015:Zombie_Process', 'os_015:ss_total', 'os_016:Agent_ping', 'os_016:Buffers_used', 'os_016:CPU_idle_pct', 'os_016:CPU_iowait_time', 'os_016:CPU_system_time', 'os_016:CPU_user_time', 'os_016:CPU_util_pct', 'os_016:Cache_used', 'os_016:Disk_avgqu_sz', 'os_016:Disk_await', 'os_016:Disk_io_util', 'os_016:Disk_rd_ios', 'os_016:Disk_rd_kbs', 'os_016:Disk_svctm', 'os_016:Disk_wr_ios', 'os_016:Disk_wr_kbs', 'os_016:FS_max_avail', 'os_016:FS_max_util', 'os_016:FS_total_space', 'os_016:FS_used_pct', 'os_016:FS_used_space', 'os_016:ICMP_ping', 'os_016:Incoming_network_traffic', 'os_016:Memory_available', 'os_016:Memory_available_pct', 'os_016:Memory_free', 'os_016:Memory_total', 'os_016:Memory_used', 'os_016:Memory_used_pct', 'os_016:Num_of_processes', 'os_016:Num_of_running_processes', 'os_016:Outgoing_network_traffic', 'os_016:Page_pi', 'os_016:Page_po', 'os_016:Processor_load_15_min', 'os_016:Processor_load_1_min', 'os_016:Processor_load_5_min', 'os_016:Received_errors_packets', 'os_016:Received_packets', 'os_016:Received_queue', 'os_016:Recv_total', 'os_016:Send_total', 'os_016:Sent_errors_packets', 'os_016:Sent_packets', 'os_016:Sent_queue', 'os_016:Shared_memory', 'os_016:Swap_used_pct', 'os_016:System_block_queue_length', 'os_016:System_wait_queue_length', 'os_016:Zombie_Process', 'os_016:ss_total', 'os_017:Agent_ping', 'os_017:Buffers_used', 'os_017:CPU_idle_pct', 'os_017:CPU_iowait_time', 'os_017:CPU_system_time', 'os_017:CPU_user_time', 'os_017:CPU_util_pct', 'os_017:Cache_used', 'os_017:Disk_avgqu_sz', 'os_017:Disk_await', 'os_017:Disk_io_util', 'os_017:Disk_rd_ios', 'os_017:Disk_rd_kbs', 'os_017:Disk_svctm', 'os_017:Disk_wr_ios', 'os_017:Disk_wr_kbs', 'os_017:FS_max_avail', 'os_017:FS_max_util', 'os_017:FS_total_space', 'os_017:FS_used_pct', 'os_017:FS_used_space', 'os_017:ICMP_ping', 'os_017:Incoming_network_traffic', 'os_017:Memory_available', 'os_017:Memory_available_pct', 'os_017:Memory_free', 'os_017:Memory_total', 'os_017:Memory_used', 'os_017:Memory_used_pct', 'os_017:Num_of_processes', 'os_017:Num_of_running_processes', 'os_017:Outgoing_network_traffic', 'os_017:Page_pi', 'os_017:Page_po', 'os_017:Processor_load_15_min', 'os_017:Processor_load_1_min', 'os_017:Processor_load_5_min', 'os_017:Received_errors_packets', 'os_017:Received_packets', 'os_017:Received_queue', 'os_017:Recv_total', 'os_017:Send_total', 'os_017:Sent_errors_packets', 'os_017:Sent_packets', 'os_017:Sent_queue', 'os_017:Shared_memory', 'os_017:Swap_used_pct', 'os_017:System_block_queue_length', 'os_017:System_wait_queue_length', 'os_017:Zombie_Process', 'os_017:ss_total', 'os_018:Agent_ping', 'os_018:Buffers_used', 'os_018:CPU_idle_pct', 'os_018:CPU_iowait_time', 'os_018:CPU_system_time', 'os_018:CPU_user_time', 'os_018:CPU_util_pct', 'os_018:Cache_used', 'os_018:Disk_avgqu_sz', 'os_018:Disk_await', 'os_018:Disk_io_util', 'os_018:Disk_rd_ios', 'os_018:Disk_rd_kbs', 'os_018:Disk_svctm', 'os_018:Disk_wr_ios', 'os_018:Disk_wr_kbs', 'os_018:FS_max_avail', 'os_018:FS_max_util', 'os_018:FS_total_space', 'os_018:FS_used_pct', 'os_018:FS_used_space', 'os_018:ICMP_ping', 'os_018:Incoming_network_traffic', 'os_018:Memory_available', 'os_018:Memory_available_pct', 'os_018:Memory_free', 'os_018:Memory_total', 'os_018:Memory_used', 'os_018:Memory_used_pct', 'os_018:Num_of_processes', 'os_018:Num_of_running_processes', 'os_018:Outgoing_network_traffic', 'os_018:Page_pi', 'os_018:Page_po', 'os_018:Processor_load_15_min', 'os_018:Processor_load_1_min', 'os_018:Processor_load_5_min', 'os_018:Received_errors_packets', 'os_018:Received_packets', 'os_018:Received_queue', 'os_018:Recv_total', 'os_018:Send_total', 'os_018:Sent_errors_packets', 'os_018:Sent_packets', 'os_018:Sent_queue', 'os_018:Shared_memory', 'os_018:Swap_used_pct', 'os_018:System_block_queue_length', 'os_018:System_wait_queue_length', 'os_018:Zombie_Process', 'os_018:ss_total', 'os_019:Agent_ping', 'os_019:Buffers_used', 'os_019:CPU_idle_pct', 'os_019:CPU_iowait_time', 'os_019:CPU_system_time', 'os_019:CPU_user_time', 'os_019:CPU_util_pct', 'os_019:Cache_used', 'os_019:Disk_avgqu_sz', 'os_019:Disk_await', 'os_019:Disk_io_util', 'os_019:Disk_rd_ios', 'os_019:Disk_rd_kbs', 'os_019:Disk_svctm', 'os_019:Disk_wr_ios', 'os_019:Disk_wr_kbs', 'os_019:FS_max_avail', 'os_019:FS_max_util', 'os_019:FS_total_space', 'os_019:FS_used_pct', 'os_019:FS_used_space', 'os_019:ICMP_ping', 'os_019:Incoming_network_traffic', 'os_019:Memory_available', 'os_019:Memory_available_pct', 'os_019:Memory_free', 'os_019:Memory_total', 'os_019:Memory_used', 'os_019:Memory_used_pct', 'os_019:Num_of_processes', 'os_019:Num_of_running_processes', 'os_019:Outgoing_network_traffic', 'os_019:Page_pi', 'os_019:Page_po', 'os_019:Processor_load_15_min', 'os_019:Processor_load_1_min', 'os_019:Processor_load_5_min', 'os_019:Received_errors_packets', 'os_019:Received_packets', 'os_019:Received_queue', 'os_019:Recv_total', 'os_019:Send_total', 'os_019:Sent_errors_packets', 'os_019:Sent_packets', 'os_019:Sent_queue', 'os_019:Shared_memory', 'os_019:Swap_used_pct', 'os_019:System_block_queue_length', 'os_019:System_wait_queue_length', 'os_019:Zombie_Process', 'os_019:ss_total', 'os_020:Agent_ping', 'os_020:Buffers_used', 'os_020:CPU_idle_pct', 'os_020:CPU_iowait_time', 'os_020:CPU_system_time', 'os_020:CPU_user_time', 'os_020:CPU_util_pct', 'os_020:Cache_used', 'os_020:Disk_avgqu_sz', 'os_020:Disk_await', 'os_020:Disk_io_util', 'os_020:Disk_rd_ios', 'os_020:Disk_rd_kbs', 'os_020:Disk_svctm', 'os_020:Disk_wr_ios', 'os_020:Disk_wr_kbs', 'os_020:FS_max_avail', 'os_020:FS_max_util', 'os_020:FS_total_space', 'os_020:FS_used_pct', 'os_020:FS_used_space', 'os_020:ICMP_ping', 'os_020:Incoming_network_traffic', 'os_020:Memory_available', 'os_020:Memory_available_pct', 'os_020:Memory_free', 'os_020:Memory_total', 'os_020:Memory_used', 'os_020:Memory_used_pct', 'os_020:Num_of_processes', 'os_020:Num_of_running_processes', 'os_020:Outgoing_network_traffic', 'os_020:Page_pi', 'os_020:Page_po', 'os_020:Processor_load_15_min', 'os_020:Processor_load_1_min', 'os_020:Processor_load_5_min', 'os_020:Received_errors_packets', 'os_020:Received_packets', 'os_020:Received_queue', 'os_020:Recv_total', 'os_020:Send_total', 'os_020:Sent_errors_packets', 'os_020:Sent_packets', 'os_020:Sent_queue', 'os_020:Shared_memory', 'os_020:Swap_used_pct', 'os_020:System_block_queue_length', 'os_020:System_wait_queue_length', 'os_020:Zombie_Process', 'os_020:ss_total', 'os_021:Agent_ping', 'os_021:Buffers_used', 'os_021:CPU_idle_pct', 'os_021:CPU_iowait_time', 'os_021:CPU_system_time', 'os_021:CPU_user_time', 'os_021:CPU_util_pct', 'os_021:Cache_used', 'os_021:Disk_avgqu_sz', 'os_021:Disk_await', 'os_021:Disk_io_util', 'os_021:Disk_rd_ios', 'os_021:Disk_rd_kbs', 'os_021:Disk_svctm', 'os_021:Disk_wr_ios', 'os_021:Disk_wr_kbs', 'os_021:FS_max_avail', 'os_021:FS_max_util', 'os_021:FS_total_space', 'os_021:FS_used_pct', 'os_021:FS_used_space', 'os_021:ICMP_ping', 'os_021:Incoming_network_traffic', 'os_021:Memory_available', 'os_021:Memory_available_pct', 'os_021:Memory_free', 'os_021:Memory_total', 'os_021:Memory_used', 'os_021:Memory_used_pct', 'os_021:Num_of_processes', 'os_021:Num_of_running_processes', 'os_021:Outgoing_network_traffic', 'os_021:Page_pi', 'os_021:Page_po', 'os_021:Processor_load_15_min', 'os_021:Processor_load_1_min', 'os_021:Processor_load_5_min', 'os_021:Received_errors_packets', 'os_021:Received_packets', 'os_021:Received_queue', 'os_021:Recv_total', 'os_021:Send_total', 'os_021:Sent_errors_packets', 'os_021:Sent_packets', 'os_021:Sent_queue', 'os_021:Shared_memory', 'os_021:Swap_used_pct', 'os_021:System_block_queue_length', 'os_021:System_wait_queue_length', 'os_021:Zombie_Process', 'os_021:ss_total', 'os_022:Agent_ping', 'os_022:Buffers_used', 'os_022:CPU_idle_pct', 'os_022:CPU_iowait_time', 'os_022:CPU_system_time', 'os_022:CPU_user_time', 'os_022:CPU_util_pct', 'os_022:Cache_used', 'os_022:Disk_avgqu_sz', 'os_022:Disk_await', 'os_022:Disk_io_util', 'os_022:Disk_rd_ios', 'os_022:Disk_rd_kbs', 'os_022:Disk_svctm', 'os_022:Disk_wr_ios', 'os_022:Disk_wr_kbs', 'os_022:FS_max_avail', 'os_022:FS_max_util', 'os_022:FS_total_space', 'os_022:FS_used_pct', 'os_022:FS_used_space', 'os_022:ICMP_ping', 'os_022:Incoming_network_traffic', 'os_022:Memory_available', 'os_022:Memory_available_pct', 'os_022:Memory_free', 'os_022:Memory_total', 'os_022:Memory_used', 'os_022:Memory_used_pct', 'os_022:Num_of_processes', 'os_022:Num_of_running_processes', 'os_022:Outgoing_network_traffic', 'os_022:Page_pi', 'os_022:Page_po', 'os_022:Processor_load_15_min', 'os_022:Processor_load_1_min', 'os_022:Processor_load_5_min', 'os_022:Received_errors_packets', 'os_022:Received_packets', 'os_022:Received_queue', 'os_022:Recv_total', 'os_022:Send_total', 'os_022:Sent_errors_packets', 'os_022:Sent_packets', 'os_022:Sent_queue', 'os_022:Shared_memory', 'os_022:Swap_used_pct', 'os_022:System_block_queue_length', 'os_022:System_wait_queue_length', 'os_022:Zombie_Process', 'os_022:ss_total', 'redis_001:Redis_key_count', 'redis_001:blocked_clients', 'redis_001:connected_clients', 'redis_001:evicted_keys', 'redis_001:expired_keys', 'redis_001:instantaneous_input_kbps', 'redis_001:instantaneous_ops_per_sec', 'redis_001:instantaneous_output_kbps', 'redis_001:keyspace_hits', 'redis_001:keyspace_misses', 'redis_001:maxmemory', 'redis_001:mem_fragmentation_ratio', 'redis_001:redis_load', 'redis_001:redis_ping', 'redis_001:rejected_connections', 'redis_001:total_commands_processed', 'redis_001:total_connections_received', 'redis_001:used_cpu_sys', 'redis_001:used_cpu_user', 'redis_001:used_memory', 'redis_001:used_memory_peak', 'redis_001:used_memory_rss', 'redis_002:Redis_key_count', 'redis_002:blocked_clients', 'redis_002:connected_clients', 'redis_002:evicted_keys', 'redis_002:expired_keys', 'redis_002:instantaneous_input_kbps', 'redis_002:instantaneous_ops_per_sec', 'redis_002:instantaneous_output_kbps', 'redis_002:keyspace_hits', 'redis_002:keyspace_misses', 'redis_002:maxmemory', 'redis_002:mem_fragmentation_ratio', 'redis_002:redis_load', 'redis_002:redis_ping', 'redis_002:rejected_connections', 'redis_002:total_commands_processed', 'redis_002:total_connections_received', 'redis_002:used_cpu_sys', 'redis_002:used_cpu_user', 'redis_002:used_memory', 'redis_002:used_memory_peak', 'redis_002:used_memory_rss', 'redis_003:Redis_key_count', 'redis_003:blocked_clients', 'redis_003:connected_clients', 'redis_003:evicted_keys', 'redis_003:expired_keys', 'redis_003:instantaneous_input_kbps', 'redis_003:instantaneous_ops_per_sec', 'redis_003:instantaneous_output_kbps', 'redis_003:keyspace_hits', 'redis_003:keyspace_misses', 'redis_003:maxmemory', 'redis_003:mem_fragmentation_ratio', 'redis_003:redis_load', 'redis_003:redis_ping', 'redis_003:rejected_connections', 'redis_003:total_commands_processed', 'redis_003:total_connections_received', 'redis_003:used_cpu_sys', 'redis_003:used_cpu_user', 'redis_003:used_memory', 'redis_003:used_memory_peak', 'redis_003:used_memory_rss', 'redis_004:Redis_key_count', 'redis_004:blocked_clients', 'redis_004:connected_clients', 'redis_004:evicted_keys', 'redis_004:expired_keys', 'redis_004:instantaneous_input_kbps', 'redis_004:instantaneous_ops_per_sec', 'redis_004:instantaneous_output_kbps', 'redis_004:keyspace_hits', 'redis_004:keyspace_misses', 'redis_004:maxmemory', 'redis_004:mem_fragmentation_ratio', 'redis_004:redis_load', 'redis_004:redis_ping', 'redis_004:rejected_connections', 'redis_004:total_commands_processed', 'redis_004:total_connections_received', 'redis_004:used_cpu_sys', 'redis_004:used_cpu_user', 'redis_004:used_memory', 'redis_004:used_memory_peak', 'redis_004:used_memory_rss', 'redis_005:Redis_key_count', 'redis_005:blocked_clients', 'redis_005:connected_clients', 'redis_005:evicted_keys', 'redis_005:expired_keys', 'redis_005:instantaneous_input_kbps', 'redis_005:instantaneous_ops_per_sec', 'redis_005:instantaneous_output_kbps', 'redis_005:keyspace_hits', 'redis_005:keyspace_misses', 'redis_005:maxmemory', 'redis_005:mem_fragmentation_ratio', 'redis_005:redis_load', 'redis_005:redis_ping', 'redis_005:rejected_connections', 'redis_005:total_commands_processed', 'redis_005:total_connections_received', 'redis_005:used_cpu_sys', 'redis_005:used_cpu_user', 'redis_005:used_memory', 'redis_005:used_memory_peak', 'redis_005:used_memory_rss', 'redis_006:Redis_key_count', 'redis_006:blocked_clients', 'redis_006:connected_clients', 'redis_006:evicted_keys', 'redis_006:expired_keys', 'redis_006:instantaneous_input_kbps', 'redis_006:instantaneous_ops_per_sec', 'redis_006:instantaneous_output_kbps', 'redis_006:keyspace_hits', 'redis_006:keyspace_misses', 'redis_006:maxmemory', 'redis_006:mem_fragmentation_ratio', 'redis_006:redis_load', 'redis_006:redis_ping', 'redis_006:rejected_connections', 'redis_006:total_commands_processed', 'redis_006:total_connections_received', 'redis_006:used_cpu_sys', 'redis_006:used_cpu_user', 'redis_006:used_memory', 'redis_006:used_memory_peak', 'redis_006:used_memory_rss', 'redis_007:Redis_key_count', 'redis_007:blocked_clients', 'redis_007:connected_clients', 'redis_007:evicted_keys', 'redis_007:expired_keys', 'redis_007:instantaneous_input_kbps', 'redis_007:instantaneous_ops_per_sec', 'redis_007:instantaneous_output_kbps', 'redis_007:keyspace_hits', 'redis_007:keyspace_misses', 'redis_007:maxmemory', 'redis_007:mem_fragmentation_ratio', 'redis_007:redis_load', 'redis_007:redis_ping', 'redis_007:rejected_connections', 'redis_007:total_commands_processed', 'redis_007:total_connections_received', 'redis_007:used_cpu_sys', 'redis_007:used_cpu_user', 'redis_007:used_memory', 'redis_007:used_memory_peak', 'redis_007:used_memory_rss', 'redis_008:Redis_key_count', 'redis_008:blocked_clients', 'redis_008:connected_clients', 'redis_008:evicted_keys', 'redis_008:expired_keys', 'redis_008:instantaneous_input_kbps', 'redis_008:instantaneous_ops_per_sec', 'redis_008:instantaneous_output_kbps', 'redis_008:keyspace_hits', 'redis_008:keyspace_misses', 'redis_008:maxmemory', 'redis_008:mem_fragmentation_ratio', 'redis_008:redis_load', 'redis_008:redis_ping', 'redis_008:rejected_connections', 'redis_008:total_commands_processed', 'redis_008:total_connections_received', 'redis_008:used_cpu_sys', 'redis_008:used_cpu_user', 'redis_008:used_memory', 'redis_008:used_memory_peak', 'redis_008:used_memory_rss', 'redis_009:Redis_key_count', 'redis_009:blocked_clients', 'redis_009:connected_clients', 'redis_009:evicted_keys', 'redis_009:expired_keys', 'redis_009:instantaneous_input_kbps', 'redis_009:instantaneous_ops_per_sec', 'redis_009:instantaneous_output_kbps', 'redis_009:keyspace_hits', 'redis_009:keyspace_misses', 'redis_009:maxmemory', 'redis_009:mem_fragmentation_ratio', 'redis_009:redis_load', 'redis_009:redis_ping', 'redis_009:rejected_connections', 'redis_009:total_commands_processed', 'redis_009:total_connections_received', 'redis_009:used_cpu_sys', 'redis_009:used_cpu_user', 'redis_009:used_memory', 'redis_009:used_memory_peak', 'redis_009:used_memory_rss', 'redis_010:Redis_key_count', 'redis_010:blocked_clients', 'redis_010:connected_clients', 'redis_010:evicted_keys', 'redis_010:expired_keys', 'redis_010:instantaneous_input_kbps', 'redis_010:instantaneous_ops_per_sec', 'redis_010:instantaneous_output_kbps', 'redis_010:keyspace_hits', 'redis_010:keyspace_misses', 'redis_010:maxmemory', 'redis_010:mem_fragmentation_ratio', 'redis_010:redis_load', 'redis_010:redis_ping', 'redis_010:rejected_connections', 'redis_010:total_commands_processed', 'redis_010:total_connections_received', 'redis_010:used_cpu_sys', 'redis_010:used_cpu_user', 'redis_010:used_memory', 'redis_010:used_memory_peak', 'redis_010:used_memory_rss', 'redis_011:Redis_key_count', 'redis_011:blocked_clients', 'redis_011:connected_clients', 'redis_011:evicted_keys', 'redis_011:expired_keys', 'redis_011:instantaneous_input_kbps', 'redis_011:instantaneous_ops_per_sec', 'redis_011:instantaneous_output_kbps', 'redis_011:keyspace_hits', 'redis_011:keyspace_misses', 'redis_011:maxmemory', 'redis_011:mem_fragmentation_ratio', 'redis_011:redis_load', 'redis_011:redis_ping', 'redis_011:rejected_connections', 'redis_011:total_commands_processed', 'redis_011:total_connections_received', 'redis_011:used_cpu_sys', 'redis_011:used_cpu_user', 'redis_011:used_memory', 'redis_011:used_memory_peak', 'redis_011:used_memory_rss', 'redis_012:Redis_key_count', 'redis_012:blocked_clients', 'redis_012:connected_clients', 'redis_012:evicted_keys', 'redis_012:expired_keys', 'redis_012:instantaneous_input_kbps', 'redis_012:instantaneous_ops_per_sec', 'redis_012:instantaneous_output_kbps', 'redis_012:keyspace_hits', 'redis_012:keyspace_misses', 'redis_012:maxmemory', 'redis_012:mem_fragmentation_ratio', 'redis_012:redis_load', 'redis_012:redis_ping', 'redis_012:rejected_connections', 'redis_012:total_commands_processed', 'redis_012:total_connections_received', 'redis_012:used_cpu_sys', 'redis_012:used_cpu_user', 'redis_012:used_memory', 'redis_012:used_memory_peak', 'redis_012:used_memory_rss'])"
      ]
     },
     "metadata": {},
     "execution_count": 40
    }
   ],
   "source": [
    "formatted_df.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "2071"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "k = list(formatted_df.keys())\n",
    "len(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "               itemid                    name  ... value        cmdb_id\n",
       "1     999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "3     999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "4     999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "6     999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "8     999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "...               ...                     ...  ...   ...            ...\n",
       "2863  999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "2864  999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "2867  999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "2869  999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "2870  999999996470837  container_fail_percent  ...   0.0  container_001\n",
       "\n",
       "[1436 rows x 6 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>itemid</th>\n      <th>name</th>\n      <th>bomc_id</th>\n      <th>timestamp</th>\n      <th>value</th>\n      <th>cmdb_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:00:21</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:01:20</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:02:19</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:03:07</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:04:08</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2863</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:55:16</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2864</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:56:15</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2867</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:57:15</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2869</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:58:15</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2870</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:59:16</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n  </tbody>\n</table>\n<p>1436 rows × 6 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 307
    }
   ],
   "source": [
    "formatted_df[k[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'container_001:container_fail_percent'"
      ]
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "k[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "               itemid                    name     bomc_id           timestamp  \\\n",
       "1     999999996470837  container_fail_percent  ZJ-004-065 2020-05-03 17:00:21   \n",
       "3     999999996470837  container_fail_percent  ZJ-004-065 2020-05-03 17:01:20   \n",
       "4     999999996470837  container_fail_percent  ZJ-004-065 2020-05-03 17:02:19   \n",
       "6     999999996470837  container_fail_percent  ZJ-004-065 2020-05-03 17:03:07   \n",
       "8     999999996470837  container_fail_percent  ZJ-004-065 2020-05-03 17:04:08   \n",
       "...               ...                     ...         ...                 ...   \n",
       "2863  999999996470837  container_fail_percent  ZJ-004-065 2020-05-04 16:55:16   \n",
       "2864  999999996470837  container_fail_percent  ZJ-004-065 2020-05-04 16:56:15   \n",
       "2867  999999996470837  container_fail_percent  ZJ-004-065 2020-05-04 16:57:15   \n",
       "2869  999999996470837  container_fail_percent  ZJ-004-065 2020-05-04 16:58:15   \n",
       "2870  999999996470837  container_fail_percent  ZJ-004-065 2020-05-04 16:59:16   \n",
       "\n",
       "      value        cmdb_id  \n",
       "1       0.0  container_001  \n",
       "3       0.0  container_001  \n",
       "4       0.0  container_001  \n",
       "6       0.0  container_001  \n",
       "8       0.0  container_001  \n",
       "...     ...            ...  \n",
       "2863    0.0  container_001  \n",
       "2864    0.0  container_001  \n",
       "2867    0.0  container_001  \n",
       "2869    0.0  container_001  \n",
       "2870    0.0  container_001  \n",
       "\n",
       "[1436 rows x 6 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>itemid</th>\n      <th>name</th>\n      <th>bomc_id</th>\n      <th>timestamp</th>\n      <th>value</th>\n      <th>cmdb_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:00:21</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:01:20</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:02:19</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:03:07</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-03 17:04:08</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2863</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:55:16</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2864</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:56:15</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2867</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:57:15</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2869</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:58:15</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n    <tr>\n      <th>2870</th>\n      <td>999999996470837</td>\n      <td>container_fail_percent</td>\n      <td>ZJ-004-065</td>\n      <td>2020-05-04 16:59:16</td>\n      <td>0.0</td>\n      <td>container_001</td>\n    </tr>\n  </tbody>\n</table>\n<p>1436 rows × 6 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 31
    }
   ],
   "source": [
    "formatte"
   ]
  }
 ]
}